require('./mongoose_service');
const axios = require('axios');
const cheerio = require('cheerio');
const redisService = require('./content_id_service');
const moment = require('moment');
const baseURL = 'https://www.acfun.cn';
const jieba = require('nodejieba');
const ArticleModel  = require('../models/acticle');



class Tag
{
    constructor(name, value, score)
    {
        this.name  = name;
        this.value = value;
        this.score = score;
    }
}
const instance = axios.create({
    baseURL: baseURL,
    timeout: 3000,
    headers: {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'}
  });

async function spideringActicles(count)
{
    const ids = await redisService.getRandomAcfunIds(count);
    let  successCount = 0;
    let errorCount = 0;
    for(let id of ids)
    {
        await getSingleArticle(id).then(r => {
            successCount++;

        }).catch(e => {
            // console.log(e);
            errorCount++ ;
            // if(e.errCode !== 4040000 ) throw e;
        });
        await new Promise(rev => {
            setTimeout(rev, 1000);
        })
    }
    return {
        successCount,
        errorCount
    }
}

async function getSingleArticle(id){
    console.log(`%s/a/ac${id}`,baseURL)
    // const client = await require('util').promisify()(url);
    // console.log(client.db);
    const res = await instance.get(`/a/ac${id}`).catch(e => {
        if(e.response && e.response.status && e.response.status == 404)
        {
            const err =  new Error('Not found');
            err.errCode = 4040000;
            throw err;
        }else{
            throw e;
        }
    });
    const html = res.data;
    // console.log(html);
    const $ = cheerio.load(html);
    const articleContent = $('#main').find('script');
    const regex = /(?<="content":").*(?="}])/;
    const dhtml = articleContent.html().match(regex)[0];
    if(!dhtml)
    {
        // if 404 ,do something
        // if removed from acfun ,do something

        // if a video,put id back to pool 
    }else{
        // add to already-got set 
        await redisService.markArticleIdSuccessed(id);
    }
    const tags = [];

    const contentHTML = `<div id="content111">${dhtml}</div>`;
    const $1 =  cheerio.load(contentHTML);
    const content = getTextOrImg('#content111',[]);
    const titleregex = /(?<="title":").*?(?=",")/;
    const title =  articleContent.html().match(titleregex)[0];
    const titleTags = jieba.extract(title,5);
    for(let obj of titleTags)
    {
        tags.push(new Tag("ARTICLE_TAG_TITLE",obj.word,obj.weight))
    }
    const originCreateAtregex = /(?<="createTime":").*?(?=",")/;
    const originCreateAt =  articleContent.html().match(originCreateAtregex)[0];
    const parentTagNameregex = /(?<="parentName":").*?(?=",")/;
    const parentTagName = articleContent.html().match(parentTagNameregex)[0];
    const nameTagregex = /(?<="name":").*?(?=",")/;
    const nameTag = articleContent.html().match(nameTagregex)[0];
    const tagListregex = /(?<="tagList":).*?(?=,"cr)/;
    const tagList = articleContent.html().match(tagListregex) ? articleContent.html().match(tagListregex)[0] : '';
    tags.push(new Tag('ARTICLE_TAG_SYS',parentTagName,1));
    tags.push(new Tag('ARTICLE_CATEGORY',nameTag,1));
    const coverUrlreg = /(?<="coverUrl":").*?(?=",")/;
    const coverUrl = articleContent.html().match(coverUrlreg) ? articleContent.html().match(coverUrlreg)[0] : '';
    const tagslist = eval(tagList);
    for(let tag of tagslist){
        tags.push(new Tag('ARTICLE_TAG_USER',tag.name,1));
    }
   
    // console.log(content);
    function getTextOrImg(dom,arr = []){
        const d = $1(dom);
        const children = d.children();
       
        if(children.length === 0)
        { 
            if(d.text())
            {
                arr.push(d.text());
            }

            if(d[0].name == 'img')
            {
                arr.push(d.attr('src'))
            }
        }else{
            children.each((index,child)=>{
                getTextOrImg(child,arr);
            })
        }
        return arr;

    }
    
    const article = {
            acfunId:id,
            content:content,
            articleContentHtml:contentHTML,
            createAt:Date.now().valueOf(),
            originCreateAt:moment(originCreateAt,"YYYY-MM-DD hh:mm:ss"),
            title:title,
            tags:tags,
            coverUrl:coverUrl
    }
    console.log(article);
    const result = await ArticleModel.model.findOneAndUpdate(
            {acfunId:id},
            {$set:article},
        {
            upsert:true,
            returnNewValue:true
        }
    );


    // await clients.db('acfun').collection('article')
    //     .findOneAndUpdate(
    //         {acfunId:id},
    //         {$set:{
    //             acfunId:id,
    //             content:content,
    //             articleContentHtml:contentHTML,
    //             createAt:Date.now().valueOf()
    //         }
            
    //     },
    //     {
    //     upsert:true,
    //     returnNewValue:true
    //     })

}





module.exports = {
    spideringActicles,
    getSingleArticle
}



   
